{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 6.5 Lab 1: Subset Selection Methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-07T12:08:06.750242Z",
     "start_time": "2023-10-07T12:08:06.666806Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import mean_squared_error,r2_score\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "from itertools import combinations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6.5.1 Best Subset Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-07T12:08:06.878099Z",
     "start_time": "2023-10-07T12:08:06.674702Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(322, 21)\n"
     ]
    },
    {
     "data": {
      "text/plain": "          Unnamed: 0  AtBat  Hits  HmRun  Runs  RBI  Walks  Years  CAtBat  \\\n0     -Andy Allanson    293    66      1    30   29     14      1     293   \n1        -Alan Ashby    315    81      7    24   38     39     14    3449   \n2       -Alvin Davis    479   130     18    66   72     76      3    1624   \n3      -Andre Dawson    496   141     20    65   78     37     11    5628   \n4  -Andres Galarraga    321    87     10    39   42     30      2     396   \n\n   CHits  ...  CRuns  CRBI  CWalks  League Division PutOuts  Assists  Errors  \\\n0     66  ...     30    29      14       A        E     446       33      20   \n1    835  ...    321   414     375       N        W     632       43      10   \n2    457  ...    224   266     263       A        W     880       82      14   \n3   1575  ...    828   838     354       N        E     200       11       3   \n4    101  ...     48    46      33       N        E     805       40       4   \n\n   Salary  NewLeague  \n0     NaN          A  \n1   475.0          N  \n2   480.0          A  \n3   500.0          N  \n4    91.5          N  \n\n[5 rows x 21 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Unnamed: 0</th>\n      <th>AtBat</th>\n      <th>Hits</th>\n      <th>HmRun</th>\n      <th>Runs</th>\n      <th>RBI</th>\n      <th>Walks</th>\n      <th>Years</th>\n      <th>CAtBat</th>\n      <th>CHits</th>\n      <th>...</th>\n      <th>CRuns</th>\n      <th>CRBI</th>\n      <th>CWalks</th>\n      <th>League</th>\n      <th>Division</th>\n      <th>PutOuts</th>\n      <th>Assists</th>\n      <th>Errors</th>\n      <th>Salary</th>\n      <th>NewLeague</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>-Andy Allanson</td>\n      <td>293</td>\n      <td>66</td>\n      <td>1</td>\n      <td>30</td>\n      <td>29</td>\n      <td>14</td>\n      <td>1</td>\n      <td>293</td>\n      <td>66</td>\n      <td>...</td>\n      <td>30</td>\n      <td>29</td>\n      <td>14</td>\n      <td>A</td>\n      <td>E</td>\n      <td>446</td>\n      <td>33</td>\n      <td>20</td>\n      <td>NaN</td>\n      <td>A</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>-Alan Ashby</td>\n      <td>315</td>\n      <td>81</td>\n      <td>7</td>\n      <td>24</td>\n      <td>38</td>\n      <td>39</td>\n      <td>14</td>\n      <td>3449</td>\n      <td>835</td>\n      <td>...</td>\n      <td>321</td>\n      <td>414</td>\n      <td>375</td>\n      <td>N</td>\n      <td>W</td>\n      <td>632</td>\n      <td>43</td>\n      <td>10</td>\n      <td>475.0</td>\n      <td>N</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>-Alvin Davis</td>\n      <td>479</td>\n      <td>130</td>\n      <td>18</td>\n      <td>66</td>\n      <td>72</td>\n      <td>76</td>\n      <td>3</td>\n      <td>1624</td>\n      <td>457</td>\n      <td>...</td>\n      <td>224</td>\n      <td>266</td>\n      <td>263</td>\n      <td>A</td>\n      <td>W</td>\n      <td>880</td>\n      <td>82</td>\n      <td>14</td>\n      <td>480.0</td>\n      <td>A</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>-Andre Dawson</td>\n      <td>496</td>\n      <td>141</td>\n      <td>20</td>\n      <td>65</td>\n      <td>78</td>\n      <td>37</td>\n      <td>11</td>\n      <td>5628</td>\n      <td>1575</td>\n      <td>...</td>\n      <td>828</td>\n      <td>838</td>\n      <td>354</td>\n      <td>N</td>\n      <td>E</td>\n      <td>200</td>\n      <td>11</td>\n      <td>3</td>\n      <td>500.0</td>\n      <td>N</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>-Andres Galarraga</td>\n      <td>321</td>\n      <td>87</td>\n      <td>10</td>\n      <td>39</td>\n      <td>42</td>\n      <td>30</td>\n      <td>2</td>\n      <td>396</td>\n      <td>101</td>\n      <td>...</td>\n      <td>48</td>\n      <td>46</td>\n      <td>33</td>\n      <td>N</td>\n      <td>E</td>\n      <td>805</td>\n      <td>40</td>\n      <td>4</td>\n      <td>91.5</td>\n      <td>N</td>\n    </tr>\n  </tbody>\n</table>\n<p>5 rows × 21 columns</p>\n</div>"
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv(r'../data/Hitters.csv')\n",
    "print(data.shape)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-07T12:08:06.909261Z",
     "start_time": "2023-10-07T12:08:06.690361Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "59"
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# counting the missing observations in salary\n",
    "data['Salary'].isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-07T12:08:06.909465Z",
     "start_time": "2023-10-07T12:08:06.695472Z"
    }
   },
   "outputs": [],
   "source": [
    "# removing or dropping the observations with null values\n",
    "data.dropna(inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-07T12:08:06.909596Z",
     "start_time": "2023-10-07T12:08:06.702026Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of data after dropiing na is  (263, 21)\n"
     ]
    }
   ],
   "source": [
    "print('Shape of data after dropiing na is ',data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-07T12:08:06.909778Z",
     "start_time": "2023-10-07T12:08:06.713902Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Qualitative variables are -  ['Unnamed: 0', 'League', 'Division', 'NewLeague']\n"
     ]
    }
   ],
   "source": [
    "qual_variables = [col for col in data.columns if data[col].dtype == 'O']\n",
    "print('Qualitative variables are - ',qual_variables)\n",
    "# The first col, unnamed 0, represents the name and is not useful for our current purpose\n",
    "# one hot encoding the qualitatve variables\n",
    "data = pd.get_dummies(data,columns = qual_variables[1:],drop_first=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-07T12:08:06.910449Z",
     "start_time": "2023-10-07T12:08:06.724542Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "   AtBat  Hits  HmRun  Runs  RBI  Walks  Years  CAtBat  CHits  CHmRun  CRuns  \\\n1    315    81      7    24   38     39     14    3449    835      69    321   \n2    479   130     18    66   72     76      3    1624    457      63    224   \n3    496   141     20    65   78     37     11    5628   1575     225    828   \n4    321    87     10    39   42     30      2     396    101      12     48   \n5    594   169      4    74   51     35     11    4408   1133      19    501   \n\n   CRBI  CWalks  PutOuts  Assists  Errors  Salary  League_N  Division_W  \\\n1   414     375      632       43      10   475.0      True        True   \n2   266     263      880       82      14   480.0     False        True   \n3   838     354      200       11       3   500.0      True       False   \n4    46      33      805       40       4    91.5      True       False   \n5   336     194      282      421      25   750.0     False        True   \n\n   NewLeague_N  \n1         True  \n2        False  \n3         True  \n4         True  \n5        False  ",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>AtBat</th>\n      <th>Hits</th>\n      <th>HmRun</th>\n      <th>Runs</th>\n      <th>RBI</th>\n      <th>Walks</th>\n      <th>Years</th>\n      <th>CAtBat</th>\n      <th>CHits</th>\n      <th>CHmRun</th>\n      <th>CRuns</th>\n      <th>CRBI</th>\n      <th>CWalks</th>\n      <th>PutOuts</th>\n      <th>Assists</th>\n      <th>Errors</th>\n      <th>Salary</th>\n      <th>League_N</th>\n      <th>Division_W</th>\n      <th>NewLeague_N</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>1</th>\n      <td>315</td>\n      <td>81</td>\n      <td>7</td>\n      <td>24</td>\n      <td>38</td>\n      <td>39</td>\n      <td>14</td>\n      <td>3449</td>\n      <td>835</td>\n      <td>69</td>\n      <td>321</td>\n      <td>414</td>\n      <td>375</td>\n      <td>632</td>\n      <td>43</td>\n      <td>10</td>\n      <td>475.0</td>\n      <td>True</td>\n      <td>True</td>\n      <td>True</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>479</td>\n      <td>130</td>\n      <td>18</td>\n      <td>66</td>\n      <td>72</td>\n      <td>76</td>\n      <td>3</td>\n      <td>1624</td>\n      <td>457</td>\n      <td>63</td>\n      <td>224</td>\n      <td>266</td>\n      <td>263</td>\n      <td>880</td>\n      <td>82</td>\n      <td>14</td>\n      <td>480.0</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>496</td>\n      <td>141</td>\n      <td>20</td>\n      <td>65</td>\n      <td>78</td>\n      <td>37</td>\n      <td>11</td>\n      <td>5628</td>\n      <td>1575</td>\n      <td>225</td>\n      <td>828</td>\n      <td>838</td>\n      <td>354</td>\n      <td>200</td>\n      <td>11</td>\n      <td>3</td>\n      <td>500.0</td>\n      <td>True</td>\n      <td>False</td>\n      <td>True</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>321</td>\n      <td>87</td>\n      <td>10</td>\n      <td>39</td>\n      <td>42</td>\n      <td>30</td>\n      <td>2</td>\n      <td>396</td>\n      <td>101</td>\n      <td>12</td>\n      <td>48</td>\n      <td>46</td>\n      <td>33</td>\n      <td>805</td>\n      <td>40</td>\n      <td>4</td>\n      <td>91.5</td>\n      <td>True</td>\n      <td>False</td>\n      <td>True</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>594</td>\n      <td>169</td>\n      <td>4</td>\n      <td>74</td>\n      <td>51</td>\n      <td>35</td>\n      <td>11</td>\n      <td>4408</td>\n      <td>1133</td>\n      <td>19</td>\n      <td>501</td>\n      <td>336</td>\n      <td>194</td>\n      <td>282</td>\n      <td>421</td>\n      <td>25</td>\n      <td>750.0</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.drop(qual_variables[0],axis = 1,inplace = True)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://xavierbourretsicotte.github.io/subset_selection.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-07T12:08:06.911168Z",
     "start_time": "2023-10-07T12:08:06.731617Z"
    }
   },
   "outputs": [],
   "source": [
    "# There is no similar  built in function as regsubset in python, therefore we will create one.\n",
    "# the reference is from the link that is mentioned above."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-07T12:08:06.911301Z",
     "start_time": "2023-10-07T12:08:06.737274Z"
    }
   },
   "outputs": [],
   "source": [
    "def fit_linear_reg(X,Y):\n",
    "    #Fit linear regression model and return RSS and R squared values\n",
    "    model_k = LinearRegression(fit_intercept = True)\n",
    "    model_k.fit(X,Y)\n",
    "    RSS = mean_squared_error(Y,model_k.predict(X)) * len(Y)\n",
    "    R_squared = model_k.score(X,Y)\n",
    "    return RSS, R_squared\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "is_executing": true,
    "ExecuteTime": {
     "start_time": "2023-10-07T12:08:06.751003Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "Loop...:   0%|          | 0/19 [00:00<?, ?it/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "d6e2f14e704d465ea2565e498a3e9b35"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from tqdm.notebook import tnrange, tqdm_notebook\n",
    "\n",
    "\n",
    "#Initialization variables\n",
    "Y = data['Salary']\n",
    "X = data.drop(columns = 'Salary', axis = 1)\n",
    "k = 19\n",
    "RSS_list, R_squared_list, feature_list = [],[], []\n",
    "numb_features = []\n",
    "\n",
    "#Looping over k = 1 to k = 11 features in X\n",
    "for k in tnrange(1,len(X.columns) + 1, desc = 'Loop...'):\n",
    "\n",
    "    #Looping over all possible combinations: from 11 choose k\n",
    "    for combo in combinations(X.columns,k):\n",
    "        tmp_result = fit_linear_reg(X[list(combo)],Y)   #Store temp result \n",
    "        RSS_list.append(tmp_result[0])                  #Append lists\n",
    "        R_squared_list.append(tmp_result[1])\n",
    "        feature_list.append(combo)\n",
    "        numb_features.append(len(combo))   \n",
    "\n",
    "#Store in DataFrame\n",
    "df = pd.DataFrame({'numb_features': numb_features,'RSS': RSS_list, 'R_squared':R_squared_list,'features':feature_list})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "# My PC was not fast enough to run the above code in even an hour, SO, i will leave it here.\n",
    "# Although what you need to do is to sort the df, on the basis of RSS, than figure out what are the num_features, with lowest\n",
    "# RSS, DO the same for R_squared."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6.5.2 Forward and Backward Stepwise Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "def forward_stepwise_selection(data,target):\n",
    "    total_features = [[]]\n",
    "    score_dict = {}\n",
    "    remaining_features = [col for col in data.columns if not col == target]\n",
    "    for i in range(1,len(data.columns)):\n",
    "        best_score = 0;best_feature = None\n",
    "        for feature in remaining_features:\n",
    "\n",
    "            X = total_features[i-1] + [feature]\n",
    "            model = LinearRegression().fit(data[X],data[target])\n",
    "            score = r2_score(data[target],model.predict(data[X]))\n",
    "#             print('For len {}, feature - {}, score is {}'.format(i,feature,score))\n",
    "\n",
    "            if score > best_score:\n",
    "                best_score = score\n",
    "                best_feature = feature\n",
    "        total_features.append(total_features[i-1] + [best_feature])\n",
    "        remaining_features.remove(best_feature)\n",
    "        score_dict[i] = best_score\n",
    "    return total_features,score_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "total_features_fwd,score_dict_fwd = forward_stepwise_selection(data,'Salary')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "# lets print the selceted features for first 5 models\n",
    "for i in range(1,len(total_features_fwd)-1):\n",
    "    print('The best model with {} features - {}'.format(i,total_features_fwd[i]))\n",
    "    print('R_2 score is ',score_dict_fwd[i])\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "# this looks a bit boring to read. Lets plot the results of the above text\n",
    "temp = pd.DataFrame({'Number of Features':np.arange(1,len(total_features_fwd)),'R_2_score':list(score_dict_fwd.values())})\n",
    "plt.figure(figsize = (12,6))\n",
    "g = sns.FacetGrid(data = temp,size=5)\n",
    "g.map(plt.scatter, 'Number of Features' , 'R_2_score')\n",
    "g.map(plt.plot, 'Number of Features', 'R_2_score')\n",
    "plt.xticks = list(np.arange(1,len(total_features_fwd)))\n",
    "plt.title('Forward Selection')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "# we can see from the above graph that as the number of features increases, the value of R2 increases.\n",
    "# If we choose this metric to choose the best model amonf all the size, we would always end up choosing the model with \n",
    "# highest features, however, we know that this is not the case, therefore we will use some other metrics to select the model."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Using cross validation to choose which model is best"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### validation approach"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "X_train,X_test,y_train,y_test = train_test_split(data.drop('Salary',axis = 1),data['Salary'],test_size = 0.2,random_state = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "validation_approach_score = {}\n",
    "for features in total_features_fwd[1:]:\n",
    "    model = LinearRegression()\n",
    "    model.fit(X_train[features],y_train)\n",
    "    score = mean_squared_error(y_test,model.predict(X_test[features])) * len(y_test)\n",
    "    validation_approach_score[len(features)] = score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "sns.lineplot(np.arange(1,len(total_features_fwd)),list(validation_approach_score.values()))\n",
    "plt.xlabel('Number of Features')\n",
    "plt.ylabel('RSS_values')\n",
    "plt.title('Validatoin Approach')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "# THe lowest value of r2 is for the model having 8 features, which is quite different from the book.\n",
    "# In the book it was the model with 10 features that was selected."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "# --OPTIONAL--\n",
    "# Lets try and use adjusted r2 to select the best modekl\n",
    "# We calcualte adjusted r2 on the training set only\n",
    "# Adj r2 = 1-(1-R2)*(n-1)/(n-p-1)\n",
    "adj_R2_score = {}\n",
    "for features in total_features_fwd[1:]:\n",
    "    model = LinearRegression()\n",
    "    model.fit(data[features],data['Salary'])\n",
    "    score = r2_score(model.predict(data[features]),data['Salary']) \n",
    "    p = len(features)\n",
    "    n = len(data)\n",
    "    adj_r2 = 1-(1-score)*(n-1)/(n-p-1)\n",
    "    adj_R2_score[len(features)] = adj_r2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "sns.lineplot(np.arange(1,len(total_features_fwd)),list(adj_R2_score.values()))\n",
    "plt.xlabel('Number of Features')\n",
    "plt.ylabel('Adj_R2_values')\n",
    "plt.title('ADJUSTED R2 VALUES')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "# Using this approach we get the best model as model with 12 features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "is_executing": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
